package cn.latenight.http;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import java.io.IOException;

public class GetHttpTest {

    public static void getHtml(){

        CloseableHttpClient httpClient=HttpClients.createDefault();
        String uri="https://bilibili.com";
        System.out.println(">>>开始爬取网页："+uri+">>>");
        HttpGet httpGet=new HttpGet(uri);
        CloseableHttpResponse httpResponse=null;
        httpGet.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36");

        try{
            httpResponse=httpClient.execute(httpGet);
            if(httpResponse.getStatusLine().getStatusCode()==200){
                System.out.println("[状态码："+httpResponse.getStatusLine().getStatusCode()+"]");
                HttpEntity httpEntity=httpResponse.getEntity();
                String html=EntityUtils.toString(httpEntity,"utf-8");
                System.out.println(">>>网页："+uri+"爬取完成>>>");
                getDemand(html);
            }else{
                System.out.println(">>>状态码："+httpResponse.getStatusLine().getStatusCode()+">>>");
                System.out.println(">>>网页："+uri+"获取失败>>>");
            }
        }catch (IOException e){
            e.printStackTrace();
        }finally {
            try {
                httpResponse.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    public static void getDemand(String html){

        System.out.println(">>>开始提取数据>>>");
        Document document=Jsoup.parse(html);
        Element element_title=document.getElementsByTag("title").first();
        System.out.println("网页标题:"+element_title.text());

        Elements elements=document.select("div.nav-name");
        for(Element element:elements){
            String nav_name=element.text();
            System.out.print(nav_name+" ");
        }
        System.out.println("\n>>>数据提取完成>>>");
    }

}
